# 图片数据爬取

#无法爬取完整的网页数据,可能数据是由ajax的方法进行加载的，使用正则表达式，主要获取的手段还是request，在数据筛中使用re正则表达式

import re
import requests
import os

if __name__ =='__main__':
    if not os.path.exists('./tupian'):
        os.mkdir('./tupian')
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.76'
    }
    url = 'https://image.baidu.com/search/albumsdata?pn&rn=30&tn=albumsdetail&word=%E5%9F%8E%E5%B8%82%E5%BB%BA%E7%AD%91%E6%91%84%E5%BD%B1%E4%B8%93%E9%A2%98&album_tab=%E5%BB%BA%E7%AD%91&album_id=7&ic=0&curPageNum'
    param = {
        'pn': '60',
        'rn': '30',
        'tn': 'albumsdetail',
        'word': '城市建筑摄影专题',
        'album_tab': '建筑',
        'album_id': '7',
        'ic': '0',
        'curPageNum': '2',
    }
    # img_data = requests.get(url= url).content
    # #text字符串，content二进制，json对象
    # with open('./tieb.jpg','wb')as fp:
    #     fp.write(img_data)
    reponse = requests.get(url=url,headers=headers,params=param).json()
    print(reponse)


    #< a class ="albumsdetail-item" href="https://image.baidu.com/search/detail?tn=baiduimagedetail&amp;word=%E5%9F%8E%E5%B8%82%E5%BB%BA%E7%AD%91%E6%91%84%E5%BD%B1%E4%B8%93%E9%A2%98&amp;album_tab=%E5%BB%BA%E7%AD%91&amp;album_id=7&amp;ie=utf-8&amp;fr=albumsdetail&amp;cs=2397542458,3133539061&amp;pi=3987&amp;pn=10&amp;ic=0&amp;objurl=https%3A%2F%2Ft7.baidu.com%2Fit%2Fu%3D2397542458%2C3133539061%26fm%3D193%26f%3DGIF"
    # target="_blank" data-index="10" width="294" style="width': '294px; height: 118px;" >',
    # < img class ="albumsdetail-item-img" src="https://t7.baidu.com/it/u=2397542458,3133539061&amp;fm=193&amp;f=GIF" style="width': '294px; height: 118px; background-color: rgb(197, 166, 142);" >',
    # < div class ="albumsdetail-item-inner-border" > < /div > < /a >
    # text1 = 'aaaaaaabbbabbabaababbabababababaaabbbba'
    # text= '<a class="albumsdetail-item" href="/search/detail?tn=baiduimagedetail&amp;word=%E5%9F%8E%E5%B8%82%E5%BB%BA%E7%AD%91%E6%91%84%E5%BD%B1%E4%B8%93%E9%A2%98&amp;album_tab=%E5%BB%BA%E7%AD%91&amp;album_id=7&amp;ie=utf-8&amp;fr=albumsdetail&amp;cs=4198287529,2774471735&amp;pi=3978&amp;pn=1&amp;ic=0&amp;objurl=https%3A%2F%2Ft7.baidu.com%2Fit%2Fu%3D4198287529%2C2774471735%26fm%3D193%26f%3DGIF" target="_blank" data-index="1" width="294" style="width': '294px; height: 196px;"><img class="albumsdetail-item-img" src="https://t7.baidu.com/it/u=4198287529,2774471735&amp;fm=193&amp;f=GIF" style="width: 294px; height: 196px; background-color: rgb(176, 162, 144);"><div class="albumsdetail-item-inner-border"></div></a>'',

    # page_text = requests.get(url=url,headers=headers).text

    # print(page_text)
    # ex = '< a class ="albumsdetail-item".*?< img class ="albumsdetail-item-img" src="(*?)" style=".*?< /a >'
    # img_src_list = re.findall(ex,page_text,re.S)
    # print(img_src_list)
    url_list = re.findall("thumbnailUrl':.*?'(.*?)'.*?, 'fromUrl",str(reponse),re.S)
    print(url_list)
    # with open('./url1.txt','w',encoding='utf-8') as fp:
    #     fp.write(url_list)
    for url_list in url_list:
        #获取图片的数据
        img_data = requests.get(url=url_list,headers=headers).content
        #给图片命名
        img_name = url_list.split('/')[-1]
        #设置图片的保存路径
        imgpath = './tupian/'+img_name
        #执行持久化存储
        with open(imgpath+'.jpg','wb')as fp:
            fp.write(img_data)
            print(img_name,'下载成功')


#对于可以分页抓取的网站，可以寻找关键变量，以下为例。注意加上循环之后，需要缩进
    # url = 'www.baidu.com/page/1/?s=11111'#原来的网址
    # url = 'www.baidu.com/page/%d/?s=11111'#寻找一个页面模板
    # for pageNum in range(1,20):
    #     new_url = format(url%pageNum)#format拼接字符串





